import { jieba } from '@/lib/jieba';

const ignoreTokens = ['分析师', '产业链', '子公司', '供应商'];
export const cachedBlocks = [];

export const tokenize = (text, wordLength = 2) => {
    if (!text) {
        return [];
    }

    const taggedWords = jieba.tag(text, true);
    // console.log(taggedWords);
    const nouns = taggedWords
        .filter(
            (word) =>
                cachedBlocks.includes(word.word) ||
                word.tag.startsWith('nz') ||
                (word.tag.startsWith('n') && word.word.length > wordLength && word.word.length < 7) ||
                (word.tag.startsWith('x') && word.word.length > wordLength && word.word.length < 7) ||
                (word.tag.startsWith('vn') && word.word.length > wordLength && word.word.length < 7) ||
                (word.tag === 'eng' && word.word.length > wordLength)
        )
        .map((word) => word.word)
        .filter((word) => !ignoreTokens.includes(word));

    if (cachedBlocks.includes(text)) nouns.push(text);

    return nouns;
};
